# from bs4 import BeautifulSoup
# import requests
import time
import CustomFunc as CF
from Settings import PATH,PROCESSESMAXNUMBER
import  Tools 
import asyncio
import multiprocessing as mp


class MySpider():
    def __init__(self, start_url=None):
        if start_url != None:
            self.start_url = start_url
            self.class_name = CF.getClassName(self.start_url)
            self.class_path = PATH + self.class_name + '/'
        else:
            self.start_url = None
            self.class_path = PATH

    def carwlGroPic(self, gro_url, gro_name=None):
        """爬取图组中的图片，使用协程异步模式"""
        if self.start_url == None:                          #兼容无参实例单独爬取图组
            self.start_url = CF.getGroStartUrl(gro_url)
            gro_name = CF.getGroName(gro_url)
            gro_path = PATH + '/' + gro_name + '/'

        gro_path = self.class_path + gro_name + '/'             
        Tools.chdir(gro_path)                               #进入保存mm图片的路径
        list = CF.getPicNameUrl(gro_url, self.start_url)            #这个list是生成器
        taskslist = []                                              #创建协程任务列表
        for pic_name, pic_url in list:
            taskslist.append(Tools.waitCarwlPic(pic_name, pic_url)) #把任务装入协程列表
        loop = asyncio.get_event_loop()                             #创建一个事件循环
        loop.run_until_complete(Tools.asy(taskslist))               #把协程任务包放入事件循环并运行

    def run(self):
        """运行爬虫，使用多进程爬取分类下的图组"""
        pool = mp.Pool(processes=PROCESSESMAXNUMBER)                #创建进程池（最大进程数）
        for gro_name, gro_url in CF.getGroNameUrl(self.start_url):
            
            pool.apply_async(                                       #装入进程池
                self.carwlGroPic,
                args=(gro_url, gro_name)
                )
        pool.close()        #关闭进程池
        pool.join()  #设置主进程阻塞，等待子进程池完成后再继续执行主进程

@Tools.timer
def main():
    #爬取全部分类图片
    # sp = MySpider(
    #     'http://www.mm131.com/qipao/'
    # )
    # sp.run()

    #爬取单个组图
    myspider = MySpider()
    myspider.carwlGroPic(
        'http://www.mm131.com/xinggan/4714.html'
    )
            
if __name__ == '__main__':
    main()
    
    